package infiews.web;

import infiews.crawler.Crawler;
import net.htmlparser.jericho.Element;
import net.htmlparser.jericho.HTMLElementName;
import net.htmlparser.jericho.LoggerProvider;
import net.htmlparser.jericho.Source;

import org.jaxen.JaxenException;
import org.jaxen.Navigator;
import org.jaxen.XPath;
import infiews.web.JerichoXPath;

import java.io.IOException;
import java.io.StringWriter;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;


// non concurrent;
public class HTMLParser {
	private Source doc;
	//private String content;
	private HashSet<String> urls;

	private static final int MAX_OUT_LINKS = Crawler.getConfig().getInt("fetcher.max_outlinks");
	
	public HTMLParser() {
		urls = new HashSet<String>();
		net.htmlparser.jericho.Config.LoggerProvider = LoggerProvider.DISABLED;
	}

	public void parse(Page page) {
		String content = page.getHTML();
		String contextURL = page.getURL();
		
		// parse html document
		doc = new Source( content );
		
			
		urls.clear();
		
		// extract urls from tags attributes href and src
		List<Element> linkElements = doc.getAllElements();
		for (Element linkElement : linkElements) {
			String href = linkElement.getAttributeValue("href");
			if( href != null ){
				href = getCanonicalURL( href, contextURL );
				if( !href.equals("") ){
					urls.add( href );
				}
			}
			String src = linkElement.getAttributeValue("src");
			if( src != null ){
				src = getCanonicalURL( src, contextURL );
				if( !src.equals("") ){
					urls.add( src );
				}
			}
		}
		//page.setHTML("");
	}
		
	private String getCanonicalURL( String href, String contextURL ){
		href = href.trim();
		if (href.length() == 0) {
			return "";
		}
		
		String hrefWithoutProtocol = href.toLowerCase();
		if (href.startsWith("http://")) {
			hrefWithoutProtocol = href.substring(7);
		}
		
		int urlCount = 0;
		if (hrefWithoutProtocol.indexOf("javascript:") < 0
				&& hrefWithoutProtocol.indexOf("@") < 0) {
			URL url = URLCanonicalizer.getCanonicalURL(href, contextURL);
			if (url != null) {
				urlCount++;
				if (urlCount > MAX_OUT_LINKS) {
					return "";
				} else {
					return url.toExternalForm();
				}
			}				
		}
		
		return "";
	}
	
	public Set<String> getLinks() {
		return urls;
	}
	
	/*public String getContent(){
		return content;
	}*/

	protected void setContentByXPathResult(Object result){
		if (result instanceof Element) {
			addToUsefulContent((Element)result);
		} else if (result instanceof List) {

			List elements = (List) result;
			for (int j = 0; j < elements.size(); j++) {
				Element element = (Element) elements.get(j);
				addToUsefulContent(element);
			}
		} else if (result instanceof String) {
			usefulContent += (String) result;
		}
	}

	protected void addToUsefulContent(Element el){
		StringWriter sw = new StringWriter();
		try{
			el.getTextExtractor().writeTo(sw);
			if(usefulContent == null){
				usefulContent = "";
			}
			usefulContent += sw.getBuffer().toString();
		}catch(IOException e){
			Crawler.log().exception(e);
			return;
		}finally{
			try{sw.close();}catch(Exception ignored){}
		}
	}

	public String getUsefulContent(){
        usefulContent  = null;
		usefulContentEstimatedLength = 0;
		
        if (Crawler.getCurrentSite().isUsingXPath()){
            Navigator navigator = DocumentNavigator.getInstance();
            Object result = null;
            try{
                XPath expr = new JerichoXPath(Crawler.getCurrentSite().getXPath(), navigator);
				result = expr.evaluate(doc);
				setContentByXPathResult(result);
            }catch(JaxenException e){
				Crawler.log().exception(e);
				usefulContent = "";
            }
        }else{
            Element body = doc.getFirstElement(HTMLElementName.BODY);
            if(body != null){
                walkChilds(body);
            }
        }
		if(usefulContent != null){
        	Crawler.log().info("Getting Useful Content : " + usefulContent.substring(0, 100));
        }
        return usefulContent;
	}

	protected boolean isContentElement(Element el){
		if(el.isEmpty()){
			return false;
		}

		if(el.getName().equals("script")){
			return false;
		}

		for(Element child : el.getAllElements()){
			if(child.equals(el)){
				continue;
			}

			if(contentTags.contains(child.getName())){
				continue;
			}else{
				return false;
			}
		}
		
		return true;
	}

	protected void walkChilds(Element el){
		for(Element child : el.getChildElements()){
			if(isContentElement(child)){
				//System.out.println(child.getDebugInfo() + " E:" + child.getTextExtractor().getEstimatedMaximumOutputLength());
				setContentRecord(child);
			}else{
				walkChilds(child);
			}
		}
	}

	protected void setContentRecord(Element el){
		if(el.getTextExtractor().getEstimatedMaximumOutputLength() < usefulContentEstimatedLength){
			return;
		}

		StringWriter sw = new StringWriter();
		try{
			el.getTextExtractor().writeTo(sw);
			if(sw.getBuffer().length() > usefulContentEstimatedLength){
				usefulContentEstimatedLength = sw.getBuffer().length();
				usefulContent = sw.getBuffer().toString();
			}
		}catch(IOException e){
			Crawler.log().exception(e);
			return;
		}finally{
			try{sw.close();}catch(Exception ignored){}
		}
	}

protected static List<String> contentTags = new ArrayList<String>();
	{	contentTags.add("br");
		contentTags.add("ul");
		contentTags.add("li");
		contentTags.add("a");
		contentTags.add("span");
		contentTags.add("strong");
		contentTags.add("i");
		contentTags.add("u");
		contentTags.add("b");
		contentTags.add("img");
		contentTags.add("h1");
		contentTags.add("h2");
		contentTags.add("h3");
		contentTags.add("h4");
		contentTags.add("h5");
		contentTags.add("h6");
		contentTags.add("p");
	}

	protected String usefulContent;
	protected long usefulContentEstimatedLength = 0;
}
